# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
import os
import json
import codecs
from itemadapter import ItemAdapter
from testPro.utils.jsonToExcel import *

class TestproPipeline:

    def open_spider(self, spider):
        with open("./testPro/conf/config.json",'r') as load_f1:
            self.config = json.load(load_f1)

        self.path = self.config["pipelines"]["outPutPath"]
        self.file = codecs.open(self.path+spider.name+'.json', 'w', encoding='utf-8')

    def close_spider(self, spider):
        self.file.close()
        jsfile = self.path+spider.name+'.json'
        excfile = self.path+spider.name+'.xlsx'
        if self.config["pipelines"]["cvs"]:
            json2excel(jsfile, excfile)
        if not self.config["pipelines"]["js"]:
            os.remove(jsfile)
        
    def process_item(self, item, spider):
        line = json.dumps(dict(item),ensure_ascii=False) + '\n'
        self.file.write(line)
        return item
